{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVD、Xavier初始化、Lipschitz常量仿真\n",
    "\n",
    "通过仿真来观测设置的函数的Lipschitz常量值，有次来观测函数稳定与否"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVD 奇异值分解\n",
    "\n",
    "*torch.svd*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 方阵分解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 2.],\n",
       "        [3., 4.]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.tensor([[1., 2],\n",
    "                  [3,  4]])\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "u,s,v = torch.svd(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.4046, -0.9145],\n",
      "        [-0.9145,  0.4046]])\n",
      "tensor([5.4650, 0.3660])\n",
      "tensor([[-0.5760,  0.8174],\n",
      "        [-0.8174, -0.5760]])\n"
     ]
    }
   ],
   "source": [
    "print(u)    # 列正交 2*2\n",
    "print(s)    # 特征值 2*2\n",
    "print(v)    # 行正交 2*2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.非方阵分解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1., 0.],\n",
       "        [0., 2., 1.]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.tensor([[1., 1, 0],\n",
    "                  [0, 2, 1]])\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "u,s,v = torch.svd(x) #usv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.4472, -0.8944],\n",
      "        [-0.8944,  0.4472]])\n",
      "tensor([2.4495, 1.0000])\n",
      "tensor([[-1.8257e-01, -8.9443e-01],\n",
      "        [-9.1287e-01,  5.9605e-08],\n",
      "        [-3.6515e-01,  4.4721e-01]])\n"
     ]
    }
   ],
   "source": [
    "print(u)    # 列正交  2*2\n",
    "print(s)    # 特征值  2*2  \n",
    "print(v)    # 列正交 (2*3 行正交) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 正交验证 \n",
    "v' * v = I"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.0000e+00, -8.9407e-08],\n",
       "        [-8.9407e-08,  1.0000e+00]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.mm(v.T, v)  # 列正交"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.8333,  0.1667, -0.3333],\n",
       "        [ 0.1667,  0.8333,  0.3333],\n",
       "        [-0.3333,  0.3333,  0.3333]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.mm(v, v.T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 初始化 Init"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.随机初始化矩阵，再进行特征值分解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2048, 4096])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(2048, 4096)   # 默认：均值为0，方差为1\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "u,s,v = torch.svd(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([109.2868, 108.7889, 108.4461,  ...,  19.2274,  19.0023,  18.9249])\n"
     ]
    }
   ],
   "source": [
    "print(s)    # 特征值：最大特征值为100多，即K0（Lipschitz常量）为100多"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.标准化：缩小方差初始化矩阵，再进行特征值分解\n",
    "\n",
    "- 标准差缩小了100倍：1->0.01\n",
    "- 特征值也缩小了100倍"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0080,  0.0127, -0.0077,  ...,  0.0123, -0.0046,  0.0017],\n",
       "        [-0.0044,  0.0073,  0.0044,  ...,  0.0088, -0.0162,  0.0144],\n",
       "        [-0.0063, -0.0021, -0.0063,  ..., -0.0001,  0.0131, -0.0017],\n",
       "        ...,\n",
       "        [ 0.0056, -0.0018, -0.0014,  ...,  0.0079, -0.0107, -0.0071],\n",
       "        [ 0.0127, -0.0041,  0.0229,  ...,  0.0023,  0.0145, -0.0081],\n",
       "        [ 0.0052,  0.0003,  0.0076,  ...,  0.0022,  0.0206, -0.0013]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.init.normal_(x, mean=0, std=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.0904, 1.0879, 1.0845,  ..., 0.1924, 0.1901, 0.1883])\n"
     ]
    }
   ],
   "source": [
    "u,s,v = torch.svd(x)\n",
    "print(s)    # 特征值：最大特征值为1多，即K0（Lipschitz常量）为1多"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Xavier初始化\n",
    "\n",
    "无论矩阵如何，最大特征值去逼近于2.0【** 鲁棒**】\n",
    "\n",
    "RMT随机理论决定\n",
    "\n",
    "每一层的Lipschitz常量都会叠加，如果有10层，那么Lipschitz常量就会达到 2^10\n",
    "\n",
    "使用：\n",
    "- xavier_uniform_\n",
    "- xavier_normal_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.9661, 1.9617, 1.9580,  ..., 0.3452, 0.3425, 0.3380])\n"
     ]
    }
   ],
   "source": [
    "nn.init.xavier_uniform_(x, gain=1)\n",
    "u,s,v = torch.svd(x)\n",
    "print(s)    # 特征值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 3.9270e-03, -1.3960e-02, -8.2770e-03,  ..., -5.1399e-03,\n",
       "          1.6307e-03, -6.9899e-03],\n",
       "        [-6.4208e-03,  5.2333e-03, -1.2042e-02,  ..., -5.3630e-04,\n",
       "          1.4566e-02, -1.2383e-02],\n",
       "        [-6.1565e-04,  5.7473e-04, -1.2923e-02,  ...,  1.7925e-03,\n",
       "          1.1649e-02, -1.3669e-02],\n",
       "        ...,\n",
       "        [-2.6337e-03, -2.9070e-03, -1.9263e-03,  ...,  2.8357e-03,\n",
       "         -1.3077e-02, -7.3394e-06],\n",
       "        [ 1.5715e-02, -5.8254e-03,  1.4319e-02,  ..., -7.6462e-03,\n",
       "          1.0943e-02,  1.2676e-04],\n",
       "        [ 1.2853e-02,  1.0038e-04, -1.6202e-02,  ...,  1.5240e-02,\n",
       "         -9.9024e-03,  4.6640e-03]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 增大矩阵大小\n",
    "x = torch.randn(10240, 10240)   # 默认：均值为0，方差为1\n",
    "nn.init.xavier_uniform_(x, gain=1)  # 初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x = x.cuda()  #放入GPU计算\n",
    "# u,s,v = torch.svd(x)\n",
    "# print(s)    # 特征值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# xavier_normal_\n",
    "# x = torch.randn(10240, 10240)   # 默认：均值为0，方差为1\n",
    "# nn.init.xavier_normal_(x, gain=1)  # 初始化\n",
    "# x = x.cuda()  #放入GPU计算\n",
    "# u,s,v = torch.svd(x)\n",
    "# print(s)    # 特征值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 神经网络初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置单个卷积层模块\n",
    "class ConvModule(nn.Module):\n",
    "    def __init__(self, \n",
    "                 in_channels, out_channels,\n",
    "                 kernel_size,       # kernel大小\n",
    "                 net_layers = 1,    # 网络层数\n",
    "                 stride = 1, padding = 0,\n",
    "                 activation = nn.ReLU()\n",
    "                 ):\n",
    "        super(ConvModule, self).__init__()\n",
    "        # 卷积层\n",
    "        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)\n",
    "        # 激活函数\n",
    "        self.activation = activation\n",
    "        # 网络层数\n",
    "        self.L = net_layers\n",
    "        # 初始化方法\n",
    "        # A.针对卷积层的权重进行初始化\n",
    "        nn.init.xavier_uniform_(self.conv.weight, gain= math.sqrt(1.0/self.L))   # 层数的导数开根\n",
    "        # B.常数初始化\n",
    "        nn.init.constant_(self.conv.bias, 0)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # 一层卷积+一层激活\n",
    "        x = self.conv(x)\n",
    "        x = self.activation(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置网络参数\n",
    "in_channels = 64\n",
    "out_channels = 64\n",
    "# 保持大小不变的卷积核\n",
    "kernel_size = 3\n",
    "stride = 1\n",
    "padding = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 一层的网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ConvModule(\n",
      "  (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (activation): ReLU()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "conv_moduel = ConvModule(in_channels, out_channels, kernel_size, net_layers = 1, stride=stride, padding=padding)\n",
    "# 打印卷积模块\n",
    "print(conv_moduel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 64, 3, 3])\n"
     ]
    }
   ],
   "source": [
    "print(conv_moduel.conv.weight.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 576])\n"
     ]
    }
   ],
   "source": [
    "print(conv_moduel.conv.weight.view(64,-1).shape)  # 将特征值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "卷积层的权重直接进行特征值分解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.2994, 1.2933, 1.2712, 1.2572, 1.2455, 1.2396, 1.2287, 1.2129, 1.2000,\n",
      "        1.1769, 1.1658, 1.1571, 1.1525, 1.1419, 1.1247, 1.1192, 1.1047, 1.0986,\n",
      "        1.0898, 1.0740, 1.0721, 1.0619, 1.0565, 1.0510, 1.0414, 1.0306, 1.0225,\n",
      "        1.0139, 0.9994, 0.9973, 0.9888, 0.9829, 0.9733, 0.9665, 0.9530, 0.9453,\n",
      "        0.9429, 0.9388, 0.9293, 0.9135, 0.9071, 0.9038, 0.8826, 0.8811, 0.8705,\n",
      "        0.8691, 0.8643, 0.8513, 0.8411, 0.8300, 0.8261, 0.8220, 0.8134, 0.8026,\n",
      "        0.7940, 0.7779, 0.7717, 0.7595, 0.7500, 0.7387, 0.7262, 0.7110, 0.7043,\n",
      "        0.6895], grad_fn=<SvdHelperBackward0>)\n"
     ]
    }
   ],
   "source": [
    "u,s,v = torch.svd(conv_moduel.conv.weight.view(64,-1))\n",
    "print(s)    # 特征值, 最大特征值1.3，即Lipschitz常数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 多层的网络\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ConvModule(\n",
      "  (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (activation): ReLU()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net_layers = 18\n",
    "conv_moduel2 = ConvModule(in_channels, out_channels, kernel_size, net_layers = net_layers, stride=stride, padding=padding)\n",
    "# 打印卷积模块\n",
    "print(conv_moduel2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.3021, 0.2986, 0.2955, 0.2941, 0.2894, 0.2864, 0.2832, 0.2816, 0.2802,\n",
      "        0.2795, 0.2776, 0.2733, 0.2708, 0.2692, 0.2679, 0.2639, 0.2608, 0.2588,\n",
      "        0.2556, 0.2546, 0.2529, 0.2515, 0.2499, 0.2490, 0.2460, 0.2416, 0.2406,\n",
      "        0.2401, 0.2385, 0.2363, 0.2348, 0.2329, 0.2304, 0.2296, 0.2277, 0.2251,\n",
      "        0.2233, 0.2216, 0.2214, 0.2169, 0.2151, 0.2136, 0.2118, 0.2110, 0.2078,\n",
      "        0.2058, 0.2048, 0.2031, 0.2007, 0.1968, 0.1948, 0.1916, 0.1897, 0.1884,\n",
      "        0.1866, 0.1835, 0.1804, 0.1789, 0.1750, 0.1733, 0.1732, 0.1702, 0.1669,\n",
      "        0.1599], grad_fn=<SvdHelperBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# 对卷积权重进行特征值分解\n",
    "u,s,v = torch.svd(conv_moduel2.conv.weight.view(64,-1))\n",
    "print(s)    # 特征值, 最大特征值0.3 = 1.3/4，即Lipschitz常数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 模拟一张图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.randn(1, 64, 5, 5)   ## 图片数，通道数，图片大小(5, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 64, 5, 5])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 经过一个卷积模块: 大小不变\n",
    "y = conv_moduel(x)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[[0.0000, 0.3442, 0.8835, 1.0153, 0.0000],\n",
       "          [0.1836, 0.3730, 0.6992, 0.6196, 0.0000],\n",
       "          [0.0000, 1.4832, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.9119, 0.0000, 0.0000, 0.4596],\n",
       "          [0.8754, 2.2422, 0.0000, 0.0000, 0.0805]],\n",
       "\n",
       "         [[0.0000, 0.4600, 0.1091, 1.4623, 0.0000],\n",
       "          [1.3724, 1.9215, 1.1899, 0.8661, 0.1451],\n",
       "          [0.0000, 0.0000, 0.0000, 0.1196, 0.1655],\n",
       "          [0.4479, 0.0000, 0.0000, 0.0000, 0.8011],\n",
       "          [0.0000, 0.3048, 0.0000, 0.0086, 0.0000]],\n",
       "\n",
       "         [[1.5873, 0.7796, 0.0000, 0.1426, 0.0000],\n",
       "          [0.0000, 0.1335, 0.5187, 0.6887, 0.2333],\n",
       "          [1.2061, 1.2022, 1.6601, 0.0000, 0.0332],\n",
       "          [0.0000, 0.4952, 0.3745, 0.0000, 0.0000],\n",
       "          [0.0000, 1.5210, 0.1140, 0.0000, 0.0000]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[0.1085, 0.1598, 0.0000, 1.0215, 0.7142],\n",
       "          [0.3135, 0.2442, 0.0000, 0.0595, 0.9551],\n",
       "          [0.3300, 0.9219, 0.1142, 0.0000, 0.3354],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0794, 1.3644],\n",
       "          [0.0000, 1.1215, 0.0000, 0.0000, 0.0000]],\n",
       "\n",
       "         [[0.0000, 0.0222, 0.0000, 0.9117, 0.0000],\n",
       "          [0.0000, 0.8505, 1.7639, 1.8208, 0.0000],\n",
       "          [0.0000, 0.8901, 1.2067, 1.3831, 0.0000],\n",
       "          [1.9900, 1.5423, 0.3710, 0.6251, 1.1261],\n",
       "          [0.0153, 0.0000, 0.2700, 0.4449, 0.0000]],\n",
       "\n",
       "         [[0.7439, 0.0000, 0.0000, 0.4496, 0.4919],\n",
       "          [0.0000, 0.3773, 0.0000, 1.2938, 0.8351],\n",
       "          [0.0000, 0.7703, 0.0000, 0.4438, 0.1097],\n",
       "          [0.0000, 0.4042, 1.5725, 0.1493, 0.6753],\n",
       "          [0.0000, 0.6989, 0.7819, 0.0000, 0.0000]]]], grad_fn=<ReluBackward0>)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[[1.2132e-02, 0.0000e+00, 0.0000e+00, 2.5657e-01, 0.0000e+00],\n",
       "          [3.1962e-01, 3.0932e-02, 1.0621e-01, 0.0000e+00, 9.0251e-02],\n",
       "          [0.0000e+00, 0.0000e+00, 4.1357e-01, 1.8441e-01, 1.0863e-01],\n",
       "          [0.0000e+00, 1.9131e-01, 1.2052e-01, 1.3310e-01, 3.3095e-01],\n",
       "          [0.0000e+00, 0.0000e+00, 5.3790e-02, 0.0000e+00, 0.0000e+00]],\n",
       "\n",
       "         [[0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 5.8231e-02],\n",
       "          [1.0991e-01, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00],\n",
       "          [0.0000e+00, 2.1134e-01, 0.0000e+00, 0.0000e+00, 2.6306e-01],\n",
       "          [5.3563e-02, 2.1307e-01, 8.1955e-02, 1.2183e-01, 1.3355e-01],\n",
       "          [3.5858e-02, 0.0000e+00, 2.8951e-01, 0.0000e+00, 0.0000e+00]],\n",
       "\n",
       "         [[0.0000e+00, 0.0000e+00, 1.9007e-02, 1.7064e-01, 1.8369e-01],\n",
       "          [0.0000e+00, 3.3433e-01, 4.6958e-03, 3.1137e-01, 0.0000e+00],\n",
       "          [0.0000e+00, 9.7303e-02, 9.4173e-02, 5.0237e-02, 3.3611e-01],\n",
       "          [0.0000e+00, 0.0000e+00, 0.0000e+00, 4.3696e-03, 1.8663e-01],\n",
       "          [5.7723e-02, 0.0000e+00, 1.5456e-01, 0.0000e+00, 0.0000e+00]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[0.0000e+00, 1.7057e-01, 4.3491e-01, 5.2095e-02, 0.0000e+00],\n",
       "          [1.2597e-01, 0.0000e+00, 6.2969e-03, 0.0000e+00, 1.1528e-01],\n",
       "          [0.0000e+00, 3.0482e-01, 0.0000e+00, 1.3488e-01, 0.0000e+00],\n",
       "          [0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00],\n",
       "          [0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00]],\n",
       "\n",
       "         [[1.3603e-01, 4.6062e-02, 0.0000e+00, 1.0360e-01, 0.0000e+00],\n",
       "          [2.6760e-01, 2.6482e-01, 2.4476e-02, 7.7330e-05, 2.7478e-01],\n",
       "          [8.3567e-02, 1.5868e-01, 1.3971e-01, 1.5771e-01, 0.0000e+00],\n",
       "          [0.0000e+00, 0.0000e+00, 0.0000e+00, 1.6754e-01, 1.3599e-01],\n",
       "          [5.2264e-02, 2.9679e-01, 3.2684e-01, 0.0000e+00, 0.0000e+00]],\n",
       "\n",
       "         [[0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 9.3630e-02],\n",
       "          [0.0000e+00, 2.9095e-02, 0.0000e+00, 2.8214e-01, 2.1615e-01],\n",
       "          [0.0000e+00, 0.0000e+00, 2.3403e-01, 7.3972e-02, 3.5853e-01],\n",
       "          [1.5780e-01, 0.0000e+00, 0.0000e+00, 2.4294e-01, 0.0000e+00],\n",
       "          [0.0000e+00, 1.8595e-01, 0.0000e+00, 0.0000e+00, 0.0000e+00]]]],\n",
       "       grad_fn=<ReluBackward0>)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 权重的初始化控制值域范围\n",
    "y2 = conv_moduel2(x)\n",
    "y2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lipschitz常量"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 线性：FC层\n",
    "\n",
    "K0 由最大特征值决定\n",
    "\n",
    "Fc: view 拉直"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.3050, 1.2733, 1.2415, 1.2379, 1.2250, 1.2186, 1.2079, 1.2019, 1.1793,\n",
      "        1.1735, 1.1671, 1.1462, 1.1349, 1.1262, 1.1210, 1.1118, 1.1075, 1.1009,\n",
      "        1.0920, 1.0870, 1.0820, 1.0669, 1.0557, 1.0487, 1.0369, 1.0273, 1.0132,\n",
      "        1.0093, 0.9979, 0.9917, 0.9878, 0.9724, 0.9712, 0.9575, 0.9517, 0.9456,\n",
      "        0.9385, 0.9269, 0.9222, 0.9133, 0.9063, 0.9031, 0.9022, 0.8930, 0.8775,\n",
      "        0.8673, 0.8621, 0.8466, 0.8411, 0.8375, 0.8280, 0.8228, 0.8085, 0.7990,\n",
      "        0.7930, 0.7778, 0.7689, 0.7597, 0.7466, 0.7395, 0.7332, 0.7140, 0.7019,\n",
      "        0.6852], grad_fn=<SvdHelperBackward0>)\n"
     ]
    }
   ],
   "source": [
    "conv = nn.Conv2d(64,64,\n",
    "                 3, stride=1, padding= 1)   # 不改变大小\n",
    "# 初始化卷积参数\n",
    "nn.init.xavier_normal_(conv.weight, gain=1.0)   #---1.0, 0.1, 1/L\n",
    "# 卷积参数SVD分解\n",
    "u,s,v = torch.svd(conv.weight.view(64,-1))\n",
    "print(s)    # 特征值, 最大值为1.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.9944, grad_fn=<MaxBackward1>)\n",
      "tensor(1.0005, grad_fn=<MaxBackward1>)\n",
      "tensor(1.0285, grad_fn=<MaxBackward1>)\n",
      "tensor(1.0341, grad_fn=<MaxBackward1>)\n",
      "tensor(1.0496, grad_fn=<MaxBackward1>)\n",
      "tensor(1.0634, grad_fn=<MaxBackward1>)\n",
      "tensor(1.0725, grad_fn=<MaxBackward1>)\n",
      "tensor(1.0725, grad_fn=<MaxBackward1>)\n"
     ]
    }
   ],
   "source": [
    "# 模拟计算Lipschitz常数：斜率的最大值\n",
    "k0 = 0\n",
    "N = 10000  # 模拟计算实验数\n",
    "for i in range(N):\n",
    "    x1 = torch.randn(10,64,10,10)  # 图片数，通道数，图片大小(10,10)\n",
    "    # 1.经过一个卷积层\n",
    "    y1 = conv(x1)\n",
    "    # 2.增加扰动后，经过一个卷积层\n",
    "    # 每个参数都增加随机扰动\n",
    "    eps = torch.randn(10,64,10,10) * 0.00001\n",
    "    x2 = x1 + eps\n",
    "    y2 = conv(x2)\n",
    "    # 3.拉直x和y\n",
    "    x1 = x1.view(64, -1)   #64*1000\n",
    "    x2 = x2.view(64, -1)\n",
    "    y1 = y1.view(64, -1)\n",
    "    y2 = y2.view(64, -1)\n",
    "    # 4.计算本次的k值\n",
    "    tmp_k0 = torch.max(torch.norm(y2-y1, p=2, dim=1) / torch.norm(x2-x1, p=2, dim=1))\n",
    "    if(tmp_k0 > k0):\n",
    "        k0= tmp_k0\n",
    "        print(tmp_k0)\n",
    "print(k0)  #1.09，逼近1.3\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LayerNorm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LayerNorm((2,), eps=1e-06, elementwise_affine=True)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = 2   # 维度: 2,32\n",
    "ln = nn.LayerNorm(d, eps=1e-6)    # ----eps=1e-6, 0\n",
    "ln"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.2456, grad_fn=<MaxBackward1>)\n",
      "tensor(0.3399, grad_fn=<MaxBackward1>)\n",
      "tensor(1.0100, grad_fn=<MaxBackward1>)\n",
      "tensor(4.4837, grad_fn=<MaxBackward1>)\n",
      "tensor(13.0425, grad_fn=<MaxBackward1>)\n",
      "tensor(32.2021, grad_fn=<MaxBackward1>)\n",
      "tensor(300.7237, grad_fn=<MaxBackward1>)\n",
      "tensor(955.9447, grad_fn=<MaxBackward1>)\n",
      "tensor(961.6682, grad_fn=<MaxBackward1>)\n",
      "tensor(974.0364, grad_fn=<MaxBackward1>)\n",
      "tensor(997.4463, grad_fn=<MaxBackward1>)\n",
      "tensor(997.4463, grad_fn=<MaxBackward1>)\n"
     ]
    }
   ],
   "source": [
    "# 模拟计算Lipschitz常数：斜率的最大值\n",
    "k0 = 0\n",
    "N = 10000  # 模拟计算实验数\n",
    "for i in range(N):\n",
    "    x1 = torch.randn(10, d)  \n",
    "    # 1.经过一个ln层\n",
    "    y1 = ln(x1)\n",
    "    # 2.增加扰动后，经过一个ln层\n",
    "    # 每个参数都增加随机扰动\n",
    "    eps = torch.randn(10, d) * 0.00001\n",
    "    x2 = x1 + eps\n",
    "    y2 = ln(x2)\n",
    "    # 3.计算本次的k值\n",
    "    tmp_k0 = torch.max(torch.norm(y2-y1, p=2, dim=1) / torch.norm(x2-x1, p=2, dim=1))\n",
    "    if(tmp_k0 > k0):\n",
    "        k0= tmp_k0\n",
    "        print(tmp_k0)\n",
    "print(k0)  #900多"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(187156.6875, grad_fn=<MaxBackward1>)\n",
      "tensor(217357.1719, grad_fn=<MaxBackward1>)\n",
      "tensor(220351.4375, grad_fn=<MaxBackward1>)\n",
      "tensor(225933.8750, grad_fn=<MaxBackward1>)\n",
      "tensor(230328.3750, grad_fn=<MaxBackward1>)\n",
      "tensor(244042.3438, grad_fn=<MaxBackward1>)\n",
      "tensor(260112.8438, grad_fn=<MaxBackward1>)\n",
      "tensor(288367.5938, grad_fn=<MaxBackward1>)\n",
      "tensor(291075.5000, grad_fn=<MaxBackward1>)\n",
      "tensor(301330.8750, grad_fn=<MaxBackward1>)\n",
      "tensor(301330.8750, grad_fn=<MaxBackward1>)\n"
     ]
    }
   ],
   "source": [
    "d = 32   # ---维度\n",
    "ln = nn.LayerNorm(d, eps=0)    # ---\n",
    "# 模拟计算Lipschitz常数：斜率的最大值\n",
    "k0 = 0\n",
    "N = 10000  # 模拟计算实验数\n",
    "for i in range(N):\n",
    "    x1 = torch.normal(mean=0, std=0.0000001, size=(10,d))   # --均值为0，方差很小\n",
    "    # 1.经过一个ln层\n",
    "    y1 = ln(x1)\n",
    "    # 2.增加扰动后，经过一个ln层\n",
    "    # 每个参数都增加随机扰动\n",
    "    eps = torch.randn(10, d) * 0.00001\n",
    "    x2 = x1 + eps\n",
    "    y2 = ln(x2)\n",
    "    # 3.计算本次的k值\n",
    "    tmp_k0 = torch.max(torch.norm(y2-y1, p=2, dim=1) / torch.norm(x2-x1, p=2, dim=1))\n",
    "    if(tmp_k0 > k0):\n",
    "        k0= tmp_k0\n",
    "        print(tmp_k0)\n",
    "print(k0)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "d2l",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
